{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 53,
   "id": "292fe05d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.core.display import display, HTML\n",
    "\n",
    "import pandas as pd\n",
    "import numpy as np # linear algebra\n",
    "import pandas as pd # data processing, CSV file I/O (e.g. pd.read_csv)\n",
    "import glob\n",
    "import os\n",
    "import gc\n",
    "\n",
    "from joblib import Parallel, delayed\n",
    "\n",
    "from sklearn import preprocessing, model_selection\n",
    "\n",
    "from sklearn.metrics import r2_score\n",
    "\n",
    "import matplotlib.pyplot as plt \n",
    "import seaborn as sns\n",
    "from tqdm.notebook import tqdm\n",
    "\n",
    "path_root = '../input/optiver-realized-volatility-prediction'\n",
    "path_data = '../input/optiver-realized-volatility-prediction'\n",
    "path_submissions = '/'\n",
    "\n",
    "target_name = 'target'\n",
    "scores_folds = {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "1efdf4be",
   "metadata": {},
   "outputs": [],
   "source": [
    "def log_return(list_stock_prices):\n",
    "    return np.log(list_stock_prices).diff() \n",
    "\n",
    "def realized_volatility(series_log_return):\n",
    "    return np.sqrt(np.sum(series_log_return**2))\n",
    "\n",
    "def rmspe(y_true, y_pred):\n",
    "    return  (np.sqrt(np.mean(np.square((y_true - y_pred) / y_true))))\n",
    "\n",
    "\n",
    "def get_stock_stat(stock_id : int, dataType = 'train'):\n",
    "    key = ['stock_id', 'time_id', 'seconds_in_bucket']\n",
    "    \n",
    "    df_book = pd.read_parquet(os.path.join(path_data, 'book_{}.parquet/stock_id={}/'.format(dataType, stock_id)))\n",
    "    df_book['stock_id'] = stock_id\n",
    "    cols = key + [col for col in df_book.columns if col not in key]\n",
    "    df_book = df_book[cols]\n",
    "    \n",
    "    df_book['wap1'] = (df_book['bid_price1'] * df_book['ask_size1'] +\n",
    "                                    df_book['ask_price1'] * df_book['bid_size1']) / (df_book['bid_size1'] + df_book['ask_size1'])\n",
    "    df_book['wap2'] = (df_book['bid_price2'] * df_book['ask_size2'] +\n",
    "                                    df_book['ask_price2'] * df_book['bid_size2']) / (df_book['bid_size2'] + df_book['ask_size2'])\n",
    "    df_book['log_return1'] = df_book.groupby(by = ['time_id'])['wap1'].apply(log_return).fillna(0)\n",
    "    df_book['log_return2'] = df_book.groupby(by = ['time_id'])['wap2'].apply(log_return).fillna(0)\n",
    "    \n",
    "    features_to_apply_realized_volatility = ['log_return'+str(i+1) for i in range(2)]\n",
    "    stock_stat = df_book.groupby(by = ['stock_id', 'time_id'])[features_to_apply_realized_volatility]\\\n",
    "                        .agg(realized_volatility).reset_index()\n",
    "\n",
    "    trade_stat =  pd.read_parquet(os.path.join(path_data,'trade_{}.parquet/stock_id={}'.format(dataType, stock_id)))\n",
    "    trade_stat = trade_stat.sort_values(by=['time_id', 'seconds_in_bucket']).reset_index(drop=True)\n",
    "    trade_stat['stock_id'] = stock_id\n",
    "    cols = key + [col for col in trade_stat.columns if col not in key]\n",
    "    trade_stat = trade_stat[cols]\n",
    "    trade_stat['trade_log_return1'] = trade_stat.groupby(by = ['time_id'])['price'].apply(log_return).fillna(0)\n",
    "    trade_stat = trade_stat.groupby(by = ['stock_id', 'time_id'])[['trade_log_return1']]\\\n",
    "                           .agg(realized_volatility).reset_index()\n",
    "\n",
    "    stock_stat = stock_stat.merge(trade_stat, on=['stock_id', 'time_id'], how='left').fillna(-999)\n",
    "    return stock_stat\n",
    "\n",
    "def get_dataSet(stock_ids : list, dataType = 'train'):\n",
    "    stock_stat = Parallel(n_jobs=-1)(\n",
    "        delayed(get_stock_stat)(stock_id, dataType) \n",
    "        for stock_id in stock_ids\n",
    "    )\n",
    "    return stock_stat_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "id": "286c23e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ac753fce28c9419aaea5465d66d55a8f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/112 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "dataType = 'train'\n",
    "all_stock_ids = train['stock_id'].unique()\n",
    "\n",
    "dds = []\n",
    "for stock_id in tqdm(all_stock_ids):\n",
    "    dd = get_stock_stat(stock_id=stock_id, dataType=dataType)\n",
    "    dds.append(dd)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 66,
   "id": "8987c6b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(os.path.join(path_data, 'train.csv'))\n",
    "dff = pd.concat(dds)\n",
    "dff = pd.merge(train, dff, on = ['stock_id', 'time_id'], how = 'left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 71,
   "id": "bef3fe0f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stock_id</th>\n",
       "      <th>time_id</th>\n",
       "      <th>log_return1</th>\n",
       "      <th>log_return2</th>\n",
       "      <th>trade_log_return1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>13</td>\n",
       "      <td>5</td>\n",
       "      <td>0.002394</td>\n",
       "      <td>0.003413</td>\n",
       "      <td>0.001721</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>13</td>\n",
       "      <td>11</td>\n",
       "      <td>0.000867</td>\n",
       "      <td>0.001349</td>\n",
       "      <td>0.000760</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>13</td>\n",
       "      <td>16</td>\n",
       "      <td>0.002077</td>\n",
       "      <td>0.002918</td>\n",
       "      <td>0.001533</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>13</td>\n",
       "      <td>31</td>\n",
       "      <td>0.001470</td>\n",
       "      <td>0.001945</td>\n",
       "      <td>0.000989</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>13</td>\n",
       "      <td>62</td>\n",
       "      <td>0.001150</td>\n",
       "      <td>0.001800</td>\n",
       "      <td>0.000857</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3824</th>\n",
       "      <td>13</td>\n",
       "      <td>32751</td>\n",
       "      <td>0.001729</td>\n",
       "      <td>0.002352</td>\n",
       "      <td>0.001428</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3825</th>\n",
       "      <td>13</td>\n",
       "      <td>32753</td>\n",
       "      <td>0.001466</td>\n",
       "      <td>0.002086</td>\n",
       "      <td>0.001061</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3826</th>\n",
       "      <td>13</td>\n",
       "      <td>32758</td>\n",
       "      <td>0.001072</td>\n",
       "      <td>0.001415</td>\n",
       "      <td>0.000748</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3827</th>\n",
       "      <td>13</td>\n",
       "      <td>32763</td>\n",
       "      <td>0.001943</td>\n",
       "      <td>0.003436</td>\n",
       "      <td>0.001751</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3828</th>\n",
       "      <td>13</td>\n",
       "      <td>32767</td>\n",
       "      <td>0.001673</td>\n",
       "      <td>0.001750</td>\n",
       "      <td>0.001492</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3829 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      stock_id  time_id  log_return1  log_return2  trade_log_return1\n",
       "0           13        5     0.002394     0.003413           0.001721\n",
       "1           13       11     0.000867     0.001349           0.000760\n",
       "2           13       16     0.002077     0.002918           0.001533\n",
       "3           13       31     0.001470     0.001945           0.000989\n",
       "4           13       62     0.001150     0.001800           0.000857\n",
       "...        ...      ...          ...          ...                ...\n",
       "3824        13    32751     0.001729     0.002352           0.001428\n",
       "3825        13    32753     0.001466     0.002086           0.001061\n",
       "3826        13    32758     0.001072     0.001415           0.000748\n",
       "3827        13    32763     0.001943     0.003436           0.001751\n",
       "3828        13    32767     0.001673     0.001750           0.001492\n",
       "\n",
       "[3829 rows x 5 columns]"
      ]
     },
     "execution_count": 71,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dds[12]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "afc5d063",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c7f2558f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 68,
   "id": "90533f3f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "import numpy as np\n",
    "from keras import backend as K"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "id": "41b10eea",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'2.5.0'"
      ]
     },
     "execution_count": 69,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tf.__version__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a6ad1dcf",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f183d620",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "69d97aa4",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "837d8ad3",
   "metadata": {},
   "outputs": [],
   "source": [
    "dataType = 'train'\n",
    "stock_id = 1\n",
    "df = get_stock_stat(stock_id=stock_id, dataType=dataType)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "b392de92",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(os.path.join(path_data, 'train.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6f1362b9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "2964877c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stock_id</th>\n",
       "      <th>time_id</th>\n",
       "      <th>log_return1</th>\n",
       "      <th>log_return2</th>\n",
       "      <th>trade_log_return1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0.004499</td>\n",
       "      <td>0.006999</td>\n",
       "      <td>0.002006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>0.001204</td>\n",
       "      <td>0.002476</td>\n",
       "      <td>0.000901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>0.002369</td>\n",
       "      <td>0.004801</td>\n",
       "      <td>0.001961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>31</td>\n",
       "      <td>0.002574</td>\n",
       "      <td>0.003637</td>\n",
       "      <td>0.001561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>62</td>\n",
       "      <td>0.001894</td>\n",
       "      <td>0.003257</td>\n",
       "      <td>0.000871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3825</th>\n",
       "      <td>0</td>\n",
       "      <td>32751</td>\n",
       "      <td>0.002579</td>\n",
       "      <td>0.003821</td>\n",
       "      <td>0.001519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3826</th>\n",
       "      <td>0</td>\n",
       "      <td>32753</td>\n",
       "      <td>0.002206</td>\n",
       "      <td>0.002847</td>\n",
       "      <td>0.001411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3827</th>\n",
       "      <td>0</td>\n",
       "      <td>32758</td>\n",
       "      <td>0.002913</td>\n",
       "      <td>0.003266</td>\n",
       "      <td>0.001521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3828</th>\n",
       "      <td>0</td>\n",
       "      <td>32763</td>\n",
       "      <td>0.003046</td>\n",
       "      <td>0.005105</td>\n",
       "      <td>0.001794</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3829</th>\n",
       "      <td>0</td>\n",
       "      <td>32767</td>\n",
       "      <td>0.001901</td>\n",
       "      <td>0.002541</td>\n",
       "      <td>0.001197</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3830 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      stock_id  time_id  log_return1  log_return2  trade_log_return1\n",
       "0            0        5     0.004499     0.006999           0.002006\n",
       "1            0       11     0.001204     0.002476           0.000901\n",
       "2            0       16     0.002369     0.004801           0.001961\n",
       "3            0       31     0.002574     0.003637           0.001561\n",
       "4            0       62     0.001894     0.003257           0.000871\n",
       "...        ...      ...          ...          ...                ...\n",
       "3825         0    32751     0.002579     0.003821           0.001519\n",
       "3826         0    32753     0.002206     0.002847           0.001411\n",
       "3827         0    32758     0.002913     0.003266           0.001521\n",
       "3828         0    32763     0.003046     0.005105           0.001794\n",
       "3829         0    32767     0.001901     0.002541           0.001197\n",
       "\n",
       "[3830 rows x 5 columns]"
      ]
     },
     "execution_count": 49,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "01801e33",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c69d63f3",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "8068a829",
   "metadata": {},
   "outputs": [],
   "source": [
    "stock_stat = df_book.groupby(by = ['stock_id', 'time_id'])[features_to_apply_realized_volatility]\\\n",
    "                    .agg(realized_volatility).reset_index()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "32fc2eb0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stock_id</th>\n",
       "      <th>time_id</th>\n",
       "      <th>log_return1</th>\n",
       "      <th>log_return2</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0.004499</td>\n",
       "      <td>0.006999</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>0.001204</td>\n",
       "      <td>0.002476</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>0.002369</td>\n",
       "      <td>0.004801</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>31</td>\n",
       "      <td>0.002574</td>\n",
       "      <td>0.003637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>62</td>\n",
       "      <td>0.001894</td>\n",
       "      <td>0.003257</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3825</th>\n",
       "      <td>0</td>\n",
       "      <td>32751</td>\n",
       "      <td>0.002579</td>\n",
       "      <td>0.003821</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3826</th>\n",
       "      <td>0</td>\n",
       "      <td>32753</td>\n",
       "      <td>0.002206</td>\n",
       "      <td>0.002847</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3827</th>\n",
       "      <td>0</td>\n",
       "      <td>32758</td>\n",
       "      <td>0.002913</td>\n",
       "      <td>0.003266</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3828</th>\n",
       "      <td>0</td>\n",
       "      <td>32763</td>\n",
       "      <td>0.003046</td>\n",
       "      <td>0.005105</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3829</th>\n",
       "      <td>0</td>\n",
       "      <td>32767</td>\n",
       "      <td>0.001901</td>\n",
       "      <td>0.002541</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3830 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      stock_id  time_id  log_return1  log_return2\n",
       "0            0        5     0.004499     0.006999\n",
       "1            0       11     0.001204     0.002476\n",
       "2            0       16     0.002369     0.004801\n",
       "3            0       31     0.002574     0.003637\n",
       "4            0       62     0.001894     0.003257\n",
       "...        ...      ...          ...          ...\n",
       "3825         0    32751     0.002579     0.003821\n",
       "3826         0    32753     0.002206     0.002847\n",
       "3827         0    32758     0.002913     0.003266\n",
       "3828         0    32763     0.003046     0.005105\n",
       "3829         0    32767     0.001901     0.002541\n",
       "\n",
       "[3830 rows x 4 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stock_stat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3a9d8fb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f1bbf69c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "bc3ca091",
   "metadata": {},
   "outputs": [],
   "source": [
    "trade_stat =  pd.read_parquet(os.path.join(path_data,'trade_{}.parquet/stock_id={}'.format(dataType, stock_id)))\n",
    "trade_stat = trade_stat.sort_values(by=['time_id', 'seconds_in_bucket']).reset_index(drop=True)\n",
    "trade_stat['stock_id'] = stock_id\n",
    "cols = key + [col for col in trade_stat.columns if col not in key]\n",
    "trade_stat = trade_stat[cols]\n",
    "trade_stat['trade_log_return1'] = trade_stat.groupby(by = ['time_id'])['price'].apply(log_return).fillna(0)\n",
    "trade_stat = trade_stat.groupby(by = ['stock_id', 'time_id'])[['trade_log_return1']]\\\n",
    "                       .agg(realized_volatility).reset_index()\n",
    "#Joining book and trade features\n",
    "stock_stat = stock_stat.merge(trade_stat, on=['stock_id', 'time_id'], how='left').fillna(-999)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "606d2a56",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "cdb017f2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c110e907",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "5c2f7cc9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stock_id</th>\n",
       "      <th>time_id</th>\n",
       "      <th>trade_log_return1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0.002006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>0.000901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>0.001961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>31</td>\n",
       "      <td>0.001561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>62</td>\n",
       "      <td>0.000871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3825</th>\n",
       "      <td>0</td>\n",
       "      <td>32751</td>\n",
       "      <td>0.001519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3826</th>\n",
       "      <td>0</td>\n",
       "      <td>32753</td>\n",
       "      <td>0.001411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3827</th>\n",
       "      <td>0</td>\n",
       "      <td>32758</td>\n",
       "      <td>0.001521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3828</th>\n",
       "      <td>0</td>\n",
       "      <td>32763</td>\n",
       "      <td>0.001794</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3829</th>\n",
       "      <td>0</td>\n",
       "      <td>32767</td>\n",
       "      <td>0.001197</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3830 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      stock_id  time_id  trade_log_return1\n",
       "0            0        5           0.002006\n",
       "1            0       11           0.000901\n",
       "2            0       16           0.001961\n",
       "3            0       31           0.001561\n",
       "4            0       62           0.000871\n",
       "...        ...      ...                ...\n",
       "3825         0    32751           0.001519\n",
       "3826         0    32753           0.001411\n",
       "3827         0    32758           0.001521\n",
       "3828         0    32763           0.001794\n",
       "3829         0    32767           0.001197\n",
       "\n",
       "[3830 rows x 3 columns]"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "trade_stat =  pd.read_parquet(os.path.join(path_data,'trade_{}.parquet/stock_id={}'.format(dataType, stock_id)))\n",
    "trade_stat = trade_stat.sort_values(by=['time_id', 'seconds_in_bucket']).reset_index(drop=True)\n",
    "trade_stat['stock_id'] = stock_id\n",
    "cols = key + [col for col in trade_stat.columns if col not in key]\n",
    "trade_stat = trade_stat[cols]\n",
    "trade_stat['trade_log_return1'] = trade_stat.groupby(by = ['time_id'])['price'].apply(log_return).fillna(0)\n",
    "trade_stat = trade_stat.groupby(by = ['stock_id', 'time_id'])[['trade_log_return1']]\\\n",
    "                               .agg(realized_volatility).reset_index()\n",
    "trade_stat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "b9508d93",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stock_id</th>\n",
       "      <th>time_id</th>\n",
       "      <th>log_return1</th>\n",
       "      <th>log_return2</th>\n",
       "      <th>trade_log_return1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0.004499</td>\n",
       "      <td>0.006999</td>\n",
       "      <td>0.002006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>0.001204</td>\n",
       "      <td>0.002476</td>\n",
       "      <td>0.000901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>0.002369</td>\n",
       "      <td>0.004801</td>\n",
       "      <td>0.001961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>31</td>\n",
       "      <td>0.002574</td>\n",
       "      <td>0.003637</td>\n",
       "      <td>0.001561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>62</td>\n",
       "      <td>0.001894</td>\n",
       "      <td>0.003257</td>\n",
       "      <td>0.000871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3825</th>\n",
       "      <td>0</td>\n",
       "      <td>32751</td>\n",
       "      <td>0.002579</td>\n",
       "      <td>0.003821</td>\n",
       "      <td>0.001519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3826</th>\n",
       "      <td>0</td>\n",
       "      <td>32753</td>\n",
       "      <td>0.002206</td>\n",
       "      <td>0.002847</td>\n",
       "      <td>0.001411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3827</th>\n",
       "      <td>0</td>\n",
       "      <td>32758</td>\n",
       "      <td>0.002913</td>\n",
       "      <td>0.003266</td>\n",
       "      <td>0.001521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3828</th>\n",
       "      <td>0</td>\n",
       "      <td>32763</td>\n",
       "      <td>0.003046</td>\n",
       "      <td>0.005105</td>\n",
       "      <td>0.001794</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3829</th>\n",
       "      <td>0</td>\n",
       "      <td>32767</td>\n",
       "      <td>0.001901</td>\n",
       "      <td>0.002541</td>\n",
       "      <td>0.001197</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3830 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      stock_id  time_id  log_return1  log_return2  trade_log_return1\n",
       "0            0        5     0.004499     0.006999           0.002006\n",
       "1            0       11     0.001204     0.002476           0.000901\n",
       "2            0       16     0.002369     0.004801           0.001961\n",
       "3            0       31     0.002574     0.003637           0.001561\n",
       "4            0       62     0.001894     0.003257           0.000871\n",
       "...        ...      ...          ...          ...                ...\n",
       "3825         0    32751     0.002579     0.003821           0.001519\n",
       "3826         0    32753     0.002206     0.002847           0.001411\n",
       "3827         0    32758     0.002913     0.003266           0.001521\n",
       "3828         0    32763     0.003046     0.005105           0.001794\n",
       "3829         0    32767     0.001901     0.002541           0.001197\n",
       "\n",
       "[3830 rows x 5 columns]"
      ]
     },
     "execution_count": 44,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stock_stat"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "8c8238b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['stock_id', 'time_id', 'seconds_in_bucket', 'price', 'size', 'order_count']"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "cols"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5482e24e",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "c3d339b5",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>stock_id</th>\n",
       "      <th>time_id</th>\n",
       "      <th>log_return1</th>\n",
       "      <th>log_return2</th>\n",
       "      <th>trade_log_return1</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>5</td>\n",
       "      <td>0.004499</td>\n",
       "      <td>0.006999</td>\n",
       "      <td>0.002006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>11</td>\n",
       "      <td>0.001204</td>\n",
       "      <td>0.002476</td>\n",
       "      <td>0.000901</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>16</td>\n",
       "      <td>0.002369</td>\n",
       "      <td>0.004801</td>\n",
       "      <td>0.001961</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>31</td>\n",
       "      <td>0.002574</td>\n",
       "      <td>0.003637</td>\n",
       "      <td>0.001561</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>62</td>\n",
       "      <td>0.001894</td>\n",
       "      <td>0.003257</td>\n",
       "      <td>0.000871</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3825</th>\n",
       "      <td>0</td>\n",
       "      <td>32751</td>\n",
       "      <td>0.002579</td>\n",
       "      <td>0.003821</td>\n",
       "      <td>0.001519</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3826</th>\n",
       "      <td>0</td>\n",
       "      <td>32753</td>\n",
       "      <td>0.002206</td>\n",
       "      <td>0.002847</td>\n",
       "      <td>0.001411</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3827</th>\n",
       "      <td>0</td>\n",
       "      <td>32758</td>\n",
       "      <td>0.002913</td>\n",
       "      <td>0.003266</td>\n",
       "      <td>0.001521</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3828</th>\n",
       "      <td>0</td>\n",
       "      <td>32763</td>\n",
       "      <td>0.003046</td>\n",
       "      <td>0.005105</td>\n",
       "      <td>0.001794</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3829</th>\n",
       "      <td>0</td>\n",
       "      <td>32767</td>\n",
       "      <td>0.001901</td>\n",
       "      <td>0.002541</td>\n",
       "      <td>0.001197</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3830 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "      stock_id  time_id  log_return1  log_return2  trade_log_return1\n",
       "0            0        5     0.004499     0.006999           0.002006\n",
       "1            0       11     0.001204     0.002476           0.000901\n",
       "2            0       16     0.002369     0.004801           0.001961\n",
       "3            0       31     0.002574     0.003637           0.001561\n",
       "4            0       62     0.001894     0.003257           0.000871\n",
       "...        ...      ...          ...          ...                ...\n",
       "3825         0    32751     0.002579     0.003821           0.001519\n",
       "3826         0    32753     0.002206     0.002847           0.001411\n",
       "3827         0    32758     0.002913     0.003266           0.001521\n",
       "3828         0    32763     0.003046     0.005105           0.001794\n",
       "3829         0    32767     0.001901     0.002541           0.001197\n",
       "\n",
       "[3830 rows x 5 columns]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "get_stock_stat(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "80c8a557",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  0,   1,   2,   3,   4,   5,   6,   7,   8,   9,  10,  11,  13,\n",
       "        14,  15,  16,  17,  18,  19,  20,  21,  22,  23,  26,  27,  28,\n",
       "        29,  30,  31,  32,  33,  34,  35,  36,  37,  38,  39,  40,  41,\n",
       "        42,  43,  44,  46,  47,  48,  50,  51,  52,  53,  55,  56,  58,\n",
       "        59,  60,  61,  62,  63,  64,  66,  67,  68,  69,  70,  72,  73,\n",
       "        74,  75,  76,  77,  78,  80,  81,  82,  83,  84,  85,  86,  87,\n",
       "        88,  89,  90,  93,  94,  95,  96,  97,  98,  99, 100, 101, 102,\n",
       "       103, 104, 105, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116,\n",
       "       118, 119, 120, 122, 123, 124, 125, 126], dtype=int64)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['stock_id'].unique()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a0be42f5",
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv(os.path.join(path_data, 'train.csv'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "f48e6fb1",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "D:\\anaconda\\lib\\site-packages\\joblib\\externals\\loky\\process_executor.py:688: UserWarning: A worker stopped while some jobs were given to the executor. This can be caused by a too short worker timeout or by a memory leak.\n",
      "  warnings.warn(\n",
      "exception calling callback for <Future at 0x210f22e61c0 state=finished raised TerminatedWorkerError>\n",
      "Traceback (most recent call last):\n",
      "  File \"D:\\anaconda\\lib\\site-packages\\joblib\\externals\\loky\\_base.py\", line 625, in _invoke_callbacks\n",
      "    callback(self)\n",
      "  File \"D:\\anaconda\\lib\\site-packages\\joblib\\parallel.py\", line 359, in __call__\n",
      "    self.parallel.dispatch_next()\n",
      "  File \"D:\\anaconda\\lib\\site-packages\\joblib\\parallel.py\", line 792, in dispatch_next\n",
      "    if not self.dispatch_one_batch(self._original_iterator):\n",
      "  File \"D:\\anaconda\\lib\\site-packages\\joblib\\parallel.py\", line 859, in dispatch_one_batch\n",
      "    self._dispatch(tasks)\n",
      "  File \"D:\\anaconda\\lib\\site-packages\\joblib\\parallel.py\", line 777, in _dispatch\n",
      "    job = self._backend.apply_async(batch, callback=cb)\n",
      "  File \"D:\\anaconda\\lib\\site-packages\\joblib\\_parallel_backends.py\", line 531, in apply_async\n",
      "    future = self._workers.submit(SafeFunction(func))\n",
      "  File \"D:\\anaconda\\lib\\site-packages\\joblib\\externals\\loky\\reusable_executor.py\", line 177, in submit\n",
      "    return super(_ReusablePoolExecutor, self).submit(\n",
      "  File \"D:\\anaconda\\lib\\site-packages\\joblib\\externals\\loky\\process_executor.py\", line 1102, in submit\n",
      "    raise self._flags.broken\n",
      "joblib.externals.loky.process_executor.TerminatedWorkerError: A worker process managed by the executor was unexpectedly terminated. This could be caused by a segmentation fault while calling the function or by an excessive memory usage causing the Operating System to kill the worker.\n",
      "\n"
     ]
    },
    {
     "ename": "TerminatedWorkerError",
     "evalue": "A worker process managed by the executor was unexpectedly terminated. This could be caused by a segmentation fault while calling the function or by an excessive memory usage causing the Operating System to kill the worker.\n",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mTerminatedWorkerError\u001b[0m                     Traceback (most recent call last)",
      "\u001b[1;32m<timed exec>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m\u001b[0m\n",
      "\u001b[1;32m<ipython-input-17-91a1e7b2a514>\u001b[0m in \u001b[0;36mget_dataSet\u001b[1;34m(stock_ids, dataType)\u001b[0m\n\u001b[0;32m     45\u001b[0m \u001b[1;32mdef\u001b[0m \u001b[0mget_dataSet\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstock_ids\u001b[0m \u001b[1;33m:\u001b[0m \u001b[0mlist\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdataType\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;34m'train'\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     46\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m---> 47\u001b[1;33m     stock_stat = Parallel(n_jobs=-1)(\n\u001b[0m\u001b[0;32m     48\u001b[0m         \u001b[0mdelayed\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mget_stock_stat\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mstock_id\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mdataType\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     49\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mstock_id\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mstock_ids\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda\\lib\\site-packages\\joblib\\parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, iterable)\u001b[0m\n\u001b[0;32m   1052\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1053\u001b[0m             \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mretrieval_context\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1054\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mretrieve\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1055\u001b[0m             \u001b[1;31m# Make sure that we get a last message telling us we are done\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1056\u001b[0m             \u001b[0melapsed_time\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtime\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtime\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;33m-\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_start_time\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda\\lib\\site-packages\\joblib\\parallel.py\u001b[0m in \u001b[0;36mretrieve\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    931\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    932\u001b[0m                 \u001b[1;32mif\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;34m'supports_timeout'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 933\u001b[1;33m                     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    934\u001b[0m                 \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    935\u001b[0m                     \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_output\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mextend\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mjob\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda\\lib\\site-packages\\joblib\\_parallel_backends.py\u001b[0m in \u001b[0;36mwrap_future_result\u001b[1;34m(future, timeout)\u001b[0m\n\u001b[0;32m    540\u001b[0m         AsyncResults.get from multiprocessing.\"\"\"\n\u001b[0;32m    541\u001b[0m         \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 542\u001b[1;33m             \u001b[1;32mreturn\u001b[0m \u001b[0mfuture\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mresult\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mtimeout\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    543\u001b[0m         \u001b[1;32mexcept\u001b[0m \u001b[0mCfTimeoutError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    544\u001b[0m             \u001b[1;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m \u001b[1;32mfrom\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda\\lib\\concurrent\\futures\\_base.py\u001b[0m in \u001b[0;36mresult\u001b[1;34m(self, timeout)\u001b[0m\n\u001b[0;32m    437\u001b[0m                 \u001b[1;32mraise\u001b[0m \u001b[0mCancelledError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    438\u001b[0m             \u001b[1;32melif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_state\u001b[0m \u001b[1;33m==\u001b[0m \u001b[0mFINISHED\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 439\u001b[1;33m                 \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m__get_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    440\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    441\u001b[0m                 \u001b[1;32mraise\u001b[0m \u001b[0mTimeoutError\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda\\lib\\concurrent\\futures\\_base.py\u001b[0m in \u001b[0;36m__get_result\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    386\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m__get_result\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    387\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_exception\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 388\u001b[1;33m             \u001b[1;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_exception\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    389\u001b[0m         \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    390\u001b[0m             \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_result\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda\\lib\\site-packages\\joblib\\externals\\loky\\_base.py\u001b[0m in \u001b[0;36m_invoke_callbacks\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    623\u001b[0m         \u001b[1;32mfor\u001b[0m \u001b[0mcallback\u001b[0m \u001b[1;32min\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_done_callbacks\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    624\u001b[0m             \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 625\u001b[1;33m                 \u001b[0mcallback\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    626\u001b[0m             \u001b[1;32mexcept\u001b[0m \u001b[0mBaseException\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    627\u001b[0m                 \u001b[0mLOGGER\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mexception\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;34m'exception calling callback for %r'\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda\\lib\\site-packages\\joblib\\parallel.py\u001b[0m in \u001b[0;36m__call__\u001b[1;34m(self, out)\u001b[0m\n\u001b[0;32m    357\u001b[0m         \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparallel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    358\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparallel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_original_iterator\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 359\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mparallel\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdispatch_next\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    360\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    361\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda\\lib\\site-packages\\joblib\\parallel.py\u001b[0m in \u001b[0;36mdispatch_next\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    790\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    791\u001b[0m         \"\"\"\n\u001b[1;32m--> 792\u001b[1;33m         \u001b[1;32mif\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mdispatch_one_batch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_original_iterator\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    793\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_iterating\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    794\u001b[0m             \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_original_iterator\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda\\lib\\site-packages\\joblib\\parallel.py\u001b[0m in \u001b[0;36mdispatch_one_batch\u001b[1;34m(self, iterator)\u001b[0m\n\u001b[0;32m    857\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[1;32mFalse\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    858\u001b[0m             \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 859\u001b[1;33m                 \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_dispatch\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mtasks\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    860\u001b[0m                 \u001b[1;32mreturn\u001b[0m \u001b[1;32mTrue\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    861\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda\\lib\\site-packages\\joblib\\parallel.py\u001b[0m in \u001b[0;36m_dispatch\u001b[1;34m(self, batch)\u001b[0m\n\u001b[0;32m    775\u001b[0m         \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_lock\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    776\u001b[0m             \u001b[0mjob_idx\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mlen\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_jobs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 777\u001b[1;33m             \u001b[0mjob\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_backend\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mapply_async\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mbatch\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m=\u001b[0m\u001b[0mcb\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    778\u001b[0m             \u001b[1;31m# A job can complete so quickly than its callback is\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    779\u001b[0m             \u001b[1;31m# called before we get here, causing self._jobs to\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda\\lib\\site-packages\\joblib\\_parallel_backends.py\u001b[0m in \u001b[0;36mapply_async\u001b[1;34m(self, func, callback)\u001b[0m\n\u001b[0;32m    529\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0mapply_async\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfunc\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mcallback\u001b[0m\u001b[1;33m=\u001b[0m\u001b[1;32mNone\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    530\u001b[0m         \u001b[1;34m\"\"\"Schedule a func to be run\"\"\"\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 531\u001b[1;33m         \u001b[0mfuture\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_workers\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0msubmit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mSafeFunction\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mfunc\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m    532\u001b[0m         \u001b[0mfuture\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mget\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mfunctools\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mpartial\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mwrap_future_result\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfuture\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    533\u001b[0m         \u001b[1;32mif\u001b[0m \u001b[0mcallback\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda\\lib\\site-packages\\joblib\\externals\\loky\\reusable_executor.py\u001b[0m in \u001b[0;36msubmit\u001b[1;34m(self, fn, *args, **kwargs)\u001b[0m\n\u001b[0;32m    175\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0msubmit\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m**\u001b[0m\u001b[0mkwargs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    176\u001b[0m         \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_submit_resize_lock\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m--> 177\u001b[1;33m             return super(_ReusablePoolExecutor, self).submit(\n\u001b[0m\u001b[0;32m    178\u001b[0m                 fn, *args, **kwargs)\n\u001b[0;32m    179\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mD:\\anaconda\\lib\\site-packages\\joblib\\externals\\loky\\process_executor.py\u001b[0m in \u001b[0;36msubmit\u001b[1;34m(self, fn, *args, **kwargs)\u001b[0m\n\u001b[0;32m   1100\u001b[0m         \u001b[1;32mwith\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_flags\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshutdown_lock\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1101\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_flags\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbroken\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mnot\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1102\u001b[1;33m                 \u001b[1;32mraise\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_flags\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mbroken\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1103\u001b[0m             \u001b[1;32mif\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_flags\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mshutdown\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1104\u001b[0m                 raise ShutdownExecutorError(\n",
      "\u001b[1;31mTerminatedWorkerError\u001b[0m: A worker process managed by the executor was unexpectedly terminated. This could be caused by a segmentation fault while calling the function or by an excessive memory usage causing the Operating System to kill the worker.\n"
     ]
    }
   ],
   "source": [
    "%time train_stock_stat_df = get_dataSet(stock_ids = train['stock_id'].unique(), dataType = 'train')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f9dae991",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
